package com.gxljc.bear.crawler.itaogao.caijing;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.itaogao.NewsBaseCrawler;
import com.gxljc.bear.crawler.itaogao.NewsBaseTable;
import com.gxljc.bear.crawler.util.HtmlUtil;
import com.gxljc.bear.crawler.util.JsoupUtil;

/**
 * 
 * @author Administrator -> haitao
 * @since 20219月20日 下午8:32:44
 */
public class CJMagazineDetailCrawler extends NewsBaseCrawler {

	/**
	 * 
	 */
	private static final long serialVersionUID = 2393229500506790378L;
	// http://magazine.caijing.com.cn/
	private static final String HOST = "http://magazine.caijing.com.cn/";
	private static final String API_URL_FORMAT = "http://app.caijing.com.cn/?app=article&controller=article&action=fulltext&contentid=%s";

	public CJMagazineDetailCrawler() {
		super("t_newspaper_caijing");
	}

	@Override
	protected List<String> genSeeds() {
		// edition_url_list
		LOG.info("--- genSeed of CaiJing ----");
		List<String> urlList = genUrlList();
		List<String> ids = getIds(urlList);
		if (!validateObject(ids)) {
			return new ArrayList<>();
		}
		LOG.info("total size=" + ids.size());
		return ids;
	}

	@SuppressWarnings("unchecked")
	private List<String> getIds(List<String> urlList) {
		if (!validateObject(urlList)) {
			return new ArrayList<>();
		}
		List<String> seeds = new LinkedList<>();
		for (String url : urlList) {
			// fetch url list by parser
			String result = "";
			Pair<String, byte[]> pair = fetResult(url, proxy);
			result = pair.first;
			if (result != null) {
				JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
				if (jsonResult != null) {
					List<String> lists = (List<String>) jsonResult.get("url_list");
					if (validateObject(lists)) {
						seeds.addAll(lists);
					} else {
						LOG.error(String.format("parse error for url=%s edition list!", url));
					}
				}
			}
		}
		return seeds;
	}

	/**
	 * 获取 财经杂志 首页种子列表
	 * 
	 * @return
	 */
	public List<String> genUrlList() {
		List<String> editions = new LinkedList<>();
		int retry = 3;
		while (retry-- > 0) {
			List<String> lists = JsoupUtil.extractAttrListByUrl(HOST, "div.fmpic a", "href");
			if (validateObject(lists)) {
				editions.addAll(lists);
				break;
			}
			LOG.info("retry=" + retry);

		}
		LOG.info("edition size=" + editions.size());
		return editions;

	}

	/**
	 * 只检测最新一期
	 */
	@Override
	protected List<String> genNewSeeds() {
		LOG.info("--- genSeed of CaiJing ----");
		List<String> urlList = genUrlList();
		LOG.info("edition(s) size=" + urlList.size());
		if (!validateObject(urlList)) {
			return new ArrayList<>();
		}
		String url = urlList.get(0);
		List<String> ids = getIds(Arrays.asList(url));
		if (!validateObject(ids)) {
			return new ArrayList<>();
		}
		LOG.info("total size=" + ids.size());
		return ids;
	}

	@SuppressWarnings("unchecked")
	@Override
	protected Pair<NewsBaseTable, byte[]> genTable(String url) {
		NewsBaseTable table = new NewsBaseTable();
		String request = url;
		String result = "";
		Pair<String, byte[]> pair = fetResult(request, proxy);
		if (pair == null) {
			return new Pair<NewsBaseTable, byte[]>(table, "".getBytes());
		}
		result = pair.first;
		byte[] html = pair.second;
		if (result != null) {
			JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
			if (jsonResult != null) {

				String title = (String) jsonResult.get("title");
				String date = (String) jsonResult.get("date");
				String author = (String) jsonResult.get("author");
				String summary = (String) jsonResult.get("summary");
				List<String> keywords = (List<String>) jsonResult.get("keywords");

				if (!StringUtil.isNullOrEmpty(title)) {
					table.setTitle(title);
				}
				if (!StringUtil.isNullOrEmpty(date)) {
					table.setPublishDate(genDate(date, "yyyy-MM-dd HH:mm:ss"));
				}
				String id = url.substring(url.lastIndexOf("/") + 1, url.lastIndexOf("."));
				Map<String, List<String>> contentMap = genContentMap(id);

				List<String> images = (List<String>) contentMap.get("pic_list");
				table = processImages(url, table, images);
				List<String> paraList = (List<String>) contentMap.get("para_list");
				String source = (String) jsonResult.get("source");
				if (validateObject(source)) {
					table.setSource(source);
				}
				if (validateObjects(paraList)) {
					table.setContent(genContent(paraList, table.getImages()));
				}
				if (validateObject(author)) {
					table.setAuthor(author);
				}
				if (validateObject(summary)) {
					table.setSummary(summary);
				}
				if (validateObject(keywords)) {
					table.setTagName(keywords);
				}
				// set default value
				table.setId(StringUtil.genMD5Val(url));
				table.setCrawlUrl(url);
				table.setCrawlDate(new Date());
				table.setCrawlFlag(1);
			}
		}
		// save2hbase(table, html);
		return new Pair<NewsBaseTable, byte[]>(table, html);
	}

	/**
	 * 
	 * @param id
	 * @return fetch 'para_list' and 'pic_list' by http post
	 */
	private Map<String, List<String>> genContentMap(String id) {
		Map<String, List<String>> map = new HashMap<>();
		String request = String.format(API_URL_FORMAT, id);
		String html = "";
		try {
			html = new String(HtmlUtil.fetch(request));
		} catch (Exception e) {
			LOG.error("fetch error for url=" + request);
			e.printStackTrace();
		}
		if (!validateObject(html)) {
			return map;
		}
		html = html.substring(1, html.length() - 2);
		// Json解析
		JSONObject object = (JSONObject) JSONObject.parse(html);
		if (!validateObject(object)) {
			return map;
		}
		String content = (String) object.get("content");
		if (!validateObject(content)) {
			return map;
		}
		// 编码转换
		String result = StringUtil.decodeUnicodeString(content);
		List<String> paras = JsoupUtil.extractList(result, "p");
		List<String> pics = JsoupUtil.extractAttrList(result, "p img", "src");
		map.put("para_list", paras);
		map.put("pic_list", pics);
		return map;
	}

	/*public static void main(String[] args) {

		 String url = "http://magazine.caijing.com.cn/20160822/4165697.shtml";
		 String id = url.substring(url.lastIndexOf("/"),
		 url.lastIndexOf("."));
		 System.out.println("id=" + id);

		 url = "http://magazine.caijing.com.cn/20160608/4130982.shtml";
		 Pair<String, byte[]> pair = new
		 CJMagazineDetailCrawler().fetResult(HOST,
		 false);
		 if (pair != null) {

		 System.out.println(pair.first);
		 }

		System.out.println(new CJMagazineDetailCrawler().genUrlList());
	}*/

}
